import config
import os
import requests 
import shutil
import time
import json

print("========= 爬虫启动!!! =========")
print("name:", config.NAME)
print("mode:", config.MODE)
print("override server", config.OVERRIDE_SERVER)
print("========= 爬虫开始!!! =========")



print("step(1/4) 下载文件")
url = config.SERVER_PREFIX +'/download' 
r = requests.post(url, {"name":config.NAME}) 
with open("./idlist.csv", "wb") as code:
    code.write(r.content)
print("好了\n")


print("step(2/4) 对齐进度")
r = requests.post(config.SERVER_PREFIX+"/getok", {"name":config.NAME})
jinDu = r.text
if jinDu != "-1":
    with open("./idlist.csv", mode="r",encoding="utf8") as fp:
        ls =  fp.readlines()
        ls = [i[:-1] for i in ls]
    if jinDu in ls:
        ls = ls[ls.index(jinDu):]
        with open("./idlist.csv", mode="w",encoding="utf8") as fp:
            ls = [i+"\n" for i in ls]
            for i in ls:
                fp.write(i)
# {} 补正
with open("./a.json", encoding="utf8", mode="r") as fp:
    a = json.load(fp)
nuId = []
for k in a.keys():
    if a[k]=={}:
        nuId.append(k)
with open("./idlist.csv", mode="r",encoding="utf8") as fp:
    ls =  fp.readlines()
    ls = [i[:-1] for i in ls]
ls.extend(nuId)
with open("./idlist.csv", mode="w",encoding="utf8") as fp:
    ls = [i+"\n" for i in ls]
    for i in ls:
        fp.write(i)

pa = True
if (len(ls)==0):
    print("似乎已经爬好了")
    pa = False
print("好了\n")


print("step(3/4) 爬取")
if pa:
    if config.MODE==2:
        os.system(config.PYTHON+" selenium2get.py")
    elif config.MODE==3:
        os.system(config.PYTHON+" selenium3get.py")
else :
    print("不用爬了,直接继续")

print("step(4/4) 储存")
ts = time.time()
ts = str(int(ts))
shutil.copyfile('./a.json', './storage/'+ts+".a.json")
shutil.copyfile('./err.json', './storage/'+ts+".err.json")